<!DOCTYPE HTML>
<html lang="zh-CN">
<head>
    <meta charset="UTF-8" />
    <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1" />
    <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1" /> 
    <title> - 天地维杰网</title>
    <meta name="keywords" content="系统架构,shutdown,不与天斗,Domino,博客,程序员,架构师,笔记,技术,分享,java,Redis">
    
    <meta property="og:title" content="">
    <meta property="og:site_name" content="天地维杰网">
    <meta property="og:image" content="/img/author.jpg"> 
    <meta name="title" content=" - 天地维杰网" />
    <meta name="description" content="天地维杰网 | 博客 | 软件 | 架构 | Java "> 
    <link rel="shortcut icon" href="http://www.shutdown.cn/img/favicon.ico" />
    <link rel="apple-touch-icon" href="http://www.shutdown.cn/img/apple-touch-icon.png" />
    <link rel="apple-touch-icon-precomposed" href="http://www.shutdown.cn/img/apple-touch-icon.png" />
    <link href="http://www.shutdown.cn/js/vendor/font-awesome/css/font-awesome.min.css?v=4.6.2" rel="stylesheet" type="text/css" />
    <link href="http://www.shutdown.cn/js/vendor/fancybox/jquery.fancybox.css?v=2.1.5" rel="stylesheet" type="text/css" />
    <link href="http://www.shutdown.cn/css/main.css" rel="stylesheet" type="text/css" />
    <link href="http://www.shutdown.cn/css/syntax.css" rel="stylesheet" type="text/css" />
    <script type="text/javascript" id="hexo.configuration">
  var NexT = window.NexT || {};
  var CONFIG = {
    scheme: 'Pisces',
    sidebar: {"position":"left","display":"post"},
     fancybox: true, 
    motion: true
  };
</script>
<script async src="https://pagead2.googlesyndication.com/pagead/js/adsbygoogle.js?client=ca-pub-7826003325059020" crossorigin="anonymous"></script>
</head>
<body itemscope itemtype="http://schema.org/WebPage" lang="zh-Hans">
<div class="container one-collumn sidebar-position-left page-home  ">
    <div class="headband"></div>

    <header id="header" class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"> <div class="site-meta  custom-logo ">

  <div class="custom-logo-site-title">
    <a href="http://www.shutdown.cn"  class="brand" rel="start">
      <span class="logo-line-before"><i></i></span>
      <span class="site-title">天地维杰网</span>
      <span class="logo-line-after"><i></i></span>
    </a>
  </div>
  <p class="site-subtitle">人如秋鸿来有信，事若春梦了无痕</p>
</div>

<div class="site-nav-toggle">
  <button>
    <span class="btn-bar"></span>
    <span class="btn-bar"></span>
    <span class="btn-bar"></span>
  </button>
</div>

<nav class="site-nav">
    <ul id="menu" class="menu">
      
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-home"></i> <br />首页
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/categories/redis/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-battery-full"></i> <br />Redis
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/categories/java/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-coffee"></i> <br />java
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/categories/linux/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-linux"></i> <br />linux
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/categories/daily/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-bug"></i> <br />日常问题
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/categories/spring/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-child"></i> <br />Spring和Springboot
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/categories/mac/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-fire"></i> <br />Mac相关
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/categories/middleware/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-gavel"></i> <br />中间件
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/categories/jiagou/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-rocket"></i> <br />架构
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/categories/python/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-ship"></i> <br />python
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/categories/front/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-bolt"></i> <br />前端
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/categories/jvm/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-balance-scale"></i> <br />jvm
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/categories/c/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-battery-empty"></i> <br />c语言
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/categories/web3/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-web3"></i> <br />web3
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/post/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-archive"></i> <br />归档
          </a>
        </li>
      
        <li class="menu-item ">
          <a href="http://www.shutdown.cn/about/" rel="section">
              <i class="menu-item-icon fa fa-fw fa-user"></i> <br />关于
          </a>
        </li>
      
      <li class="menu-item menu-item-search">
        <a href="javascript:;" class="popup-trigger"> <i class="menu-item-icon fa fa-search fa-fw"></i> <br /> 搜索</a>
      </li>
    </ul>
    <div class="site-search">
      <div class="popup">
 <span class="search-icon fa fa-search"></span>
 <input type="text" id="local-search-input">
 <div id="local-search-result"></div>
 <span class="popup-btn-close">close</span>
</div>

    </div>
</nav>

 </div>
    </header>

    <main id="main" class="main">
      <div class="main-inner">
        <div class="content-wrap">
          <div id="content" class="content">
            
<section id="posts" class="posts-expand">
  <article class="post post-type-normal " itemscope itemtype="http://schema.org/Article">
    <header class="post-header">
      <h1 class="post-title" itemprop="name headline">
        <a class="post-title-link" href="http://www.shutdown.cn/post/redis%E9%9B%86%E7%BE%A4%E5%AE%9E%E4%BE%8B%E5%86%85%E5%AD%98%E4%BD%BF%E7%94%A8%E7%8E%87%E9%A3%99%E5%8D%87%E6%8E%92%E6%9F%A5/" itemprop="url">
        
        </a>
      </h1>
      <div class="post-meta">
      <span class="post-time">
<span class="post-meta-item-icon">
    <i class="fa fa-calendar-o"></i>
</span>
<span class="post-meta-item-text">时间：</span>
<time itemprop="dateCreated" datetime="2016-03-22T13:04:35+08:00" content="0001-01-01">
    0001-01-01
</time>
</span> 
      
      
       <span>
&nbsp; | &nbsp;
<span class="post-meta-item-icon">
    <i class="fa fa-eye"></i>
</span>
<span class="post-meta-item-text">阅读：</span>
<span class="leancloud-visitors-count">157 字 ~1分钟</span>
</span>
      </div>
    </header>
    <div class="post-body" itemprop="articleBody">
    

    <p>在一个阳光明媚的下午，突然生产环境有一个缓存实例发出内存使用率超过90%的告警，然后立刻和小伙伴们一起看是什么情况。</p>

<p>现象是这样的，集群里的一个实例的内存使用率超过了90%，而这个实例的从节点，内存使用率却很低。而且其他分片的内存使用率都很低，只有这个分片高。见下图cachecloud实例状态图。</p>

<div  align="center"><img src="https://wejack639.oss-cn-beijing.aliyuncs.com/blogimages/img/20220531185619.png" width = 900 /> </div>

<p>首先想到的肯定是大key导致的集群倾斜了，于是就先把这个分片的数据dump出来，然后使用rdbtools工具分析下dump文件，把大key找出来，再联系业务人员。</p>

<p>在做rdb分析的同时，也考虑到，会不会是有人偷偷连接了实例，然后使用了monitor忘记关掉，于是另外登录到该实例所在的服务器并连接实例后使用<code>client list</code>命令查看客户端的连接，看是否有带<code>monitor</code>相关命令的。发现并没有。</p>
<div class="highlight"><pre style="background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-fallback" data-lang="fallback">~]$ redis-cli -p 9999 client list | grep monitor
~]$ </code></pre></div>
<p>另一方面使用连接实例使用<code>bgsave</code>命令生成dump数据文件的同事也发现，生成的数据文件的大小只有130多M，分析虽然也分析出来了一个大key，但是大小也只有130多M，是一个拥有360多万个元素的<code>List</code>。</p>

<div  align="center"><img src="https://wejack639.oss-cn-beijing.aliyuncs.com/blogimages/img/20220531192746.png" width = 900 /> </div>

<p>但是这应该完全不足以造成9个G的内存使用量。我还以为是bgsave命令没正常执行？我又重新执行了一次bgsave命令，查看执行时间和dump数据确认确实是130多M。使用info命令查看实例的信息，也看不出问题。</p>

<p>一筹莫展之际，同事亮哥从实例的<code>client list</code>命令的结果看出了所以然。</p>

<div  align="center"><img src="https://wejack639.oss-cn-beijing.aliyuncs.com/blogimages/img/20220531194128.png" width = 900 /> </div>
<div class="highlight"><pre style="background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-fallback" data-lang="fallback">]$ redis-cli -p 9012 client list | grep -v &#34;omem=0&#34; 
id=11164727 addr=10.4.7.48:50478 fd=458 name= age=4230 idle=303 flags=c db=0 sub=0 psub=0 multi=-1 qbuf=163072064 qbuf-free=0 obl=0 oll=659943 omem=8546641352 events=rw cmd=lrange</code></pre></div>
<p>这个被筛选出来的客户端链接，<code>omem</code>怎么有8个G。<code>omem</code>是什么值。它是redis服务对客户端连接输出缓存的内存占用量。</p>

<p>查看redis实例的慢日志，可以看到如下的结果，相同List key的lrange查询，参数是0 -1。</p>

<div  align="center"><img src="https://wejack639.oss-cn-beijing.aliyuncs.com/blogimages/img/20220531212315.png" width = 900 /> </div>

<p>经过思考，大概可以这样理解，使用这个缓存的实例有一个<code>List</code>类型的大key，有360多万个元素，大概是130多M，然后客户端频繁的通过 <code>lrange keyname 0 -1</code>的方式全量获取所有元素。缓存服务端接收到指令，耗时700000多微秒（700多毫秒）把对应的数据给扔到输出缓冲区中，然后接着处理别的指令去了，然后因为这个指令调用的很频繁，输出缓冲区的数据就越积越多，但客户端接收数据的速度有限。形象点可以理解为一个池子一边接水一边放水，放水速度小于接水速度，接水的就是Redis服务处理指令获取到的结果，放水的就是客户端接收返回结果，这个池子就是输出缓存区，也是要占用Redis实例的内存的，时间足够，池子迟早会满。这也就是Redis实例内存不断增长的原因。</p>

<div  align="center"><img src="https://wejack639.oss-cn-beijing.aliyuncs.com/blogimages/img/20220531212501.png" width = 900 /> </div>

<p>因为Redis实例内的数据的数量并没有改变，所以该实例的Slave实例的内存并没有变化。</p>

<p>回过头来看，如果master实例的内存暴涨，但是slave实例的内存与master实例的内存差距很大的情况下，就很可能是monitor命令未关或者输出缓冲区积压导致的。</p>

<p>最终我们手动将对应的客户端连接kill掉，然后内存唰的一下就降下来了。</p>
<div class="highlight"><pre style="background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-fallback" data-lang="fallback">~]$ redis-cli -p 9012 client list | grep -v &#34;omem=0&#34;
id=2253936 addr=10.4.7.48:52303 fd=276 name= age=7580 idle=36 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=1581 omem=20931976 events=rw cmd=lrange
id=2252693 addr=10.4.7.48:51712 fd=252 name= age=9268 idle=230 flags=c db=0 sub=0 psub=0 multi=-1 qbuf=79914221 qbuf-free=0 obl=0 oll=504978 omem=6682230512 events=rw cmd=lrange
id=2258097 addr=10.4.7.48:54497 fd=345 name= age=1172 idle=194 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=254068 omem=3361306536 events=rw cmd=lrange
id=2257345 addr=10.4.7.48:54063 fd=401 name= age=2438 idle=167 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=1642 omem=21713784 events=rw cmd=lrange
~]$ redis-cli -p 9012 client kill 10.4.7.48:51712
OK
~]$ redis-cli -p 9012 client kill 10.4.7.48:54497
OK</code></pre></div>
<div  align="center"><img src="https://wejack639.oss-cn-beijing.aliyuncs.com/blogimages/img/20220531214007.png" width = 900 /> </div>

    </div>
    <footer class="post-footer">
     

     <div class="post-nav">
    <div class="post-nav-next post-nav-item">
    
        <a href="http://www.shutdown.cn/post/redis%E8%BF%9B%E7%A8%8B%E7%BB%91%E5%AE%9A%E6%8C%87%E5%AE%9A%E7%9A%84cpu%E6%A0%B8/" rel="next" title="">
        <i class="fa fa-chevron-left"></i> 
        </a>
    
    </div>

    <div class="post-nav-prev post-nav-item">
    
        <a href="http://www.shutdown.cn/post/redis%E9%9B%AA%E5%B4%A9%E6%95%88%E5%BA%94%E7%9A%84%E5%8E%9F%E5%9B%A0%E5%8F%8A%E8%A7%A3%E5%86%B3%E6%96%B9%E6%A1%88/" rel="prev" title="">
         <i class="fa fa-chevron-right"></i>
        </a>
    
    </div>
</div>
      
     
     
     






    </footer>
  </article>
</section>

          </div>
        </div>
        <div class="sidebar-toggle">
  <div class="sidebar-toggle-line-wrap">
    <span class="sidebar-toggle-line sidebar-toggle-line-first"></span>
    <span class="sidebar-toggle-line sidebar-toggle-line-middle"></span>
    <span class="sidebar-toggle-line sidebar-toggle-line-last"></span>
  </div>
</div>
<aside id="sidebar" class="sidebar">
  <div class="sidebar-inner">

    <section class="site-overview sidebar-panel  sidebar-panel-active ">
      <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
    <img class="site-author-image" itemprop="image"
        src="http://www.shutdown.cn/img/author.jpg"
        alt="不与天斗Domino" />
    <p class="site-author-name" itemprop="name">不与天斗Domino</p>
    <p class="site-description motion-element" itemprop="description"> 
        Programmer &amp; Architect</p>
</div>
      <nav class="site-state motion-element">
    <div class="site-state-item site-state-posts">
      <a href="http://www.shutdown.cn/post/">
        <span class="site-state-item-count">183</span>
        <span class="site-state-item-name">日志</span>
      </a>
    </div>
    <div class="site-state-item site-state-categories">    
        <a href="http://www.shutdown.cn/categories/">      
         
        <span class="site-state-item-count">15</span>
        
        <span class="site-state-item-name">分类</span>
        
        </a>
    </div>

    <div class="site-state-item site-state-tags">
        <a href="http://www.shutdown.cn/tags/">
         
        <span class="site-state-item-count">224</span>
        
        <span class="site-state-item-name">标签</span>
        </a>
    </div>
</nav>
      
      

      

      <div class="links-of-blogroll motion-element inline">
<script type="text/javascript" src="//rf.revolvermaps.com/0/0/8.js?i=&amp;m=0&amp;s=220&amp;c=ff0000&amp;cr1=ffffff&amp;f=arial&amp;l=33&amp;bv=35" async="async"></script>
</div>

    </section>
    
  </div>
</aside>

      </div>
    </main>
   
    <footer id="footer" class="footer">
      <div class="footer-inner">
        <div class="copyright" >
  <span itemprop="copyrightYear">  &copy; 
  2013 - 2023</span>
  <span class="with-love"><i class="fa fa-heart"></i></span>
  <span class="author" itemprop="copyrightHolder">天地维杰网</span>
  <span class="icp" itemprop="copyrightHolder"><a href="https://beian.miit.gov.cn/" target="_blank">京ICP备13019191号-1</a></span>
</div>
<div class="powered-by">
  Powered by - <a class="theme-link" href="http://gohugo.io" target="_blank" title="hugo" >Hugo v0.63.2</a>
</div>
<div class="theme-info">
  Theme by - <a class="theme-link" href="https://github.com/xtfly/hugo-theme-next" target="_blank"> NexT
  </a>
</div>


      </div>
    </footer>

    <div class="back-to-top">
      <i class="fa fa-arrow-up"></i>
      <span id="scrollpercent"><span>0</span>%</span>
    </div>
  </div>

  

<script type="text/javascript">
  if (Object.prototype.toString.call(window.Promise) !== '[object Function]') {
    window.Promise = null;
  }
</script>
<script type="text/javascript" src="http://www.shutdown.cn/js/vendor/jquery/index.js?v=2.1.3"></script>
<script type="text/javascript" src="http://www.shutdown.cn/js/vendor/fastclick/lib/fastclick.min.js?v=1.0.6"></script> 
<script type="text/javascript" src="http://www.shutdown.cn/js/vendor/jquery_lazyload/jquery.lazyload.js?v=1.9.7"></script>
<script type="text/javascript" src="http://www.shutdown.cn/js/vendor/velocity/velocity.min.js?v=1.2.1"></script>
<script type="text/javascript" src="http://www.shutdown.cn/js/vendor/velocity/velocity.ui.min.js?v=1.2.1"></script>
<script src="http://www.shutdown.cn/js/vendor/ua-parser-js/dist/ua-parser.min.js?v=0.7.9"></script>

<script src="http://www.shutdown.cn/js/vendor/fancybox/jquery.fancybox.pack.js?v=2.1.5"></script>

<script type="text/javascript" src="http://www.shutdown.cn/js/utils.js"></script>
<script type="text/javascript" src="http://www.shutdown.cn/js/motion.js"></script>
<script type="text/javascript" src="http://www.shutdown.cn/js/affix.js"></script>
<script type="text/javascript" src="http://www.shutdown.cn/js/schemes/pisces.js"></script>

<script type="text/javascript" src="http://www.shutdown.cn/js/scrollspy.js"></script>
<script type="text/javascript" src="http://www.shutdown.cn/js/post-details.js"></script>
<script type="text/javascript" src="http://www.shutdown.cn/js/toc.js"></script>

<script type="text/javascript" src="http://www.shutdown.cn/js/bootstrap.js"></script>

<script type="text/javascript" src="http://www.shutdown.cn/js/search.js"></script>
<script type="text/x-mathjax-config">
  MathJax.Hub.Config({
    extensions: ["tex2jax.js"],
    jax: ["input/TeX", "output/HTML-CSS"],
    tex2jax: {
      inlineMath: [ ['$','$'] ],
      displayMath: [ ['$$','$$'] ],
      processEscapes: true
    },
    "HTML-CSS": { fonts: ["TeX"] }
  });
</script>
<script src='https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_HTMLorMML' async></script>
</body>
</html>